READ - I've created a python script to allow the user to click the button and then they can either have all of the underyling code shown, OR they can just look at the raw output (charts, plots, whatever).
As you know, sometimes these notebooks contain a fair amount of code... and sometimes folks just want the results... here is an example
Code defaults to NOT showing any code, so click the toggle button to view the show the underlying code...
# %load toggle.py
# allows code to be hidden, unhidden on jupyter notebook
from IPython.display import HTML
HTML('''<script>
function code_toggle() {
if (code_shown){
$('div.input').hide('500');
$('#toggleButton').val('Show Code')
} else {
$('div.input').show('500');
$('#toggleButton').val('Hide All Notebook Code')
}
code_shown = !code_shown
}
$( document ).ready(function(){
code_shown=false;
$('div.input').hide()
});</script><form action="javascript:code_toggle()"><input type="submit" id="toggleButton" value="Show All Notebook Code"></form>''')
Import standard python libraries
import warnings
warnings.filterwarnings("ignore")
# import matplotlib.cbook
# warnings.filterwarnings("ignore",category=matplotlib.cbook.mplDeprecation)
import pickle
import datetime
import multiprocessing as mp
import pandas as pd
pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
plt.style.use('dark_background')
# default start, but later you can change if you desire
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.neighbors import LocalOutlierFactor
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
Move directories to read in file
%cd D:\ABSOLUTELY_CRITICAL_BACKUP_FOLDER_LOCATION\
D:\ABSOLUTELY_CRITICAL_BACKUP_FOLDER_LOCATION
Read in data
pf = pd.read_pickle("SSS.pkl")
# this pickle file was previously created from millions of rows
# of data and then filtered for removal of any NaN values,
# extensively cleaned and reproduced
# --- Data Time Domain ---
# Date Range: March 2018 - March 2021
# Granularity: 25s
# Nodes: Single Node Location
Plot snipper for understanding
pf.head(10)
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2018-03-06 22:14:56 | apds_9006_020 | intensity | 3.537000 |
| 2018-03-06 22:14:56 | apds_9006_020 | intensity | 3.537000 |
| 2018-03-06 22:14:56 | hih6130 | humidity | 61.830002 |
| 2018-03-06 22:14:56 | hih6130 | humidity | 61.830002 |
| 2018-03-06 22:14:56 | hih6130 | temperature | 9.620000 |
| 2018-03-06 22:14:56 | hih6130 | temperature | 9.620000 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_x | -148.182007 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_x | -150.000000 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_y | -1603.635986 |
| 2018-03-06 22:14:56 | hmc5883l | magnetic_field_y | -1604.545044 |
Show number of unique sensors and sensor parameters being recorded:
print('\nThe individual sensors present at this location/node:\n')
c=1
for s in pf.sensor.unique():
print(c,") ", s)
c = c+1
The individual sensors present at this location/node: 1 ) apds_9006_020 2 ) hih6130 3 ) hmc5883l 4 ) ml8511 5 ) mlx75305 6 ) tmp421 7 ) tsl250rd 8 ) tsl260rd 9 ) bmp180 10 ) hih4030 11 ) htu21d 12 ) mma8452q 13 ) pr103j2 14 ) spv1840lr5h_b 15 ) tmp112 16 ) tsys01 17 ) at0 18 ) at1 19 ) at2 20 ) at3 21 ) lps25h 22 ) sht25 23 ) co 24 ) h2s 25 ) no2 26 ) o3 27 ) oxidizing_gases 28 ) reducing_gases 29 ) so2
print('\nThe sensor parameters being monitored:\n')
for p in pf.parameter.unique():print(p) # params
The sensor parameters being monitored: intensity humidity temperature magnetic_field_x magnetic_field_y magnetic_field_z pressure acceleration_x acceleration_y acceleration_z concentration
# rebuild at categorical for speed
pf['sensor'] = pf.sensor.astype('category')
pf['parameter'] = pf.parameter.astype('category')
# pf.dtypes
# sensor category
# parameter category
# value_hrf float32
# dtype: object
print('\nNumber of rows of raw data:\n', len(pf))
# 82M rows... 82 406 931
Number of rows of raw data: 82406931
print('\nNumber of rows of data ........', f'{len(pf):,}')
Number of rows of data ........ 82,406,931
# date range
print('\nStart .....', pf.head(1).index[0])
print('\nEnd .......', pf.tail(1).index[0])
Start ..... 2018-03-06 22:14:56 End ....... 2021-03-22 05:18:42
print("\nRange of the dates for this particular dataset:")
pf.tail(1).index[0] - pf.head(1).index[0]
# over a thousand days (3 years) of 25s granularity data, not bad
Range of the dates for this particular dataset:
Timedelta('1111 days 07:03:46')
# stop: you can re-use this code up to here to
# to spin off multiple sensor analysis if desired
# this is an excellent base dataframe to do more additional examination...
# but for now we will grab particular window
Grab window in 2019
We will grab a particular window in 2019 of just over two months.
We still have a fair amount of data (given sensors record every 25s, and we have many many sensors at this particular location)
start_date = '2019-01-01'
end_date = '2019-02-28'
minidf = pf.loc[start_date:end_date].copy()
# our base window will be a wide 2 month window to get
# the core dbscan algorithm working properly
# fyi, len(minidf) # 4,883,544 roughly 5M rows of data
minidf.head(49)
# so at this point, i have a single node, and
# will grab a single sensor and build
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | apds_9006_020 | intensity | 0.723000 |
| 2019-01-01 00:00:03 | hih6130 | humidity | 24.290001 |
| 2019-01-01 00:00:03 | hih6130 | temperature | 28.660000 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_x | -155.455002 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_y | -1548.182007 |
| 2019-01-01 00:00:03 | hmc5883l | magnetic_field_z | -521.429016 |
| 2019-01-01 00:00:03 | ml8511 | intensity | 46.351002 |
| 2019-01-01 00:00:03 | mlx75305 | intensity | 4.242000 |
| 2019-01-01 00:00:03 | tmp421 | temperature | 9.000000 |
| 2019-01-01 00:00:03 | tsl250rd | intensity | 23.672001 |
| 2019-01-01 00:00:03 | tsl260rd | intensity | 26.113001 |
| 2019-01-01 00:00:03 | bmp180 | pressure | 1020.489990 |
| 2019-01-01 00:00:03 | bmp180 | temperature | 7.300000 |
| 2019-01-01 00:00:03 | hih4030 | humidity | 83.050003 |
| 2019-01-01 00:00:03 | htu21d | humidity | 88.540001 |
| 2019-01-01 00:00:03 | htu21d | temperature | 4.830000 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_x | -2.930000 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_y | -998.046997 |
| 2019-01-01 00:00:03 | mma8452q | acceleration_z | -0.977000 |
| 2019-01-01 00:00:03 | pr103j2 | temperature | 4.350000 |
| 2019-01-01 00:00:03 | spv1840lr5h_b | intensity | 70.699997 |
| 2019-01-01 00:00:03 | tmp112 | temperature | 4.880000 |
| 2019-01-01 00:00:03 | tsl250rd | intensity | 0.000000 |
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.800000 |
| 2019-01-01 00:00:28 | apds_9006_020 | intensity | 0.804000 |
| 2019-01-01 00:00:28 | hih6130 | humidity | 24.290001 |
| 2019-01-01 00:00:28 | hih6130 | temperature | 28.660000 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_x | -158.182007 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_y | -1550.909058 |
| 2019-01-01 00:00:28 | hmc5883l | magnetic_field_z | -504.082001 |
| 2019-01-01 00:00:28 | ml8511 | intensity | 46.370998 |
| 2019-01-01 00:00:28 | mlx75305 | intensity | 4.554000 |
| 2019-01-01 00:00:28 | tmp421 | temperature | 9.000000 |
| 2019-01-01 00:00:28 | tsl250rd | intensity | 23.674000 |
| 2019-01-01 00:00:28 | tsl260rd | intensity | 26.115000 |
| 2019-01-01 00:00:28 | bmp180 | pressure | 1020.510010 |
| 2019-01-01 00:00:28 | bmp180 | temperature | 7.300000 |
| 2019-01-01 00:00:28 | hih4030 | humidity | 83.349998 |
| 2019-01-01 00:00:28 | htu21d | humidity | 88.309998 |
| 2019-01-01 00:00:28 | htu21d | temperature | 4.760000 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_x | 0.977000 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_y | -998.046997 |
| 2019-01-01 00:00:28 | mma8452q | acceleration_z | 8.789000 |
| 2019-01-01 00:00:28 | pr103j2 | temperature | 4.250000 |
| 2019-01-01 00:00:28 | spv1840lr5h_b | intensity | 68.760002 |
| 2019-01-01 00:00:28 | tmp112 | temperature | 4.810000 |
| 2019-01-01 00:00:28 | tsl250rd | intensity | 0.000000 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.720000 |
| 2019-01-01 00:00:53 | apds_9006_020 | intensity | 0.643000 |
Filter by parameter of temperature
# important
# grabbing the right accurate sensor for this was tricky
# i am focusing for now on the single dimension of temperature
tdf = minidf[minidf.parameter == 'temperature'].copy()
# you could go thru all the sensors and plot here,
# elements like temperature or whatever
tdf.head(10)
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | hih6130 | temperature | 28.66 |
| 2019-01-01 00:00:03 | tmp421 | temperature | 9.00 |
| 2019-01-01 00:00:03 | bmp180 | temperature | 7.30 |
| 2019-01-01 00:00:03 | htu21d | temperature | 4.83 |
| 2019-01-01 00:00:03 | pr103j2 | temperature | 4.35 |
| 2019-01-01 00:00:03 | tmp112 | temperature | 4.88 |
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | hih6130 | temperature | 28.66 |
| 2019-01-01 00:00:28 | tmp421 | temperature | 9.00 |
| 2019-01-01 00:00:28 | bmp180 | temperature | 7.30 |
Filter to on a single sensor for now (temp only parameter)
tsys = tdf[tdf.sensor == 'tsys01'].copy()
# the specific tsys01 sub-sensor for now
tsys.head(10)
# len(tsys) # 203 481
# so about 200,000 rows of data...
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:00:53 | tsys01 | temperature | 4.67 |
| 2019-01-01 00:01:18 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:01:43 | tsys01 | temperature | 4.71 |
| 2019-01-01 00:02:08 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:02:33 | tsys01 | temperature | 4.63 |
| 2019-01-01 00:02:58 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:03:23 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:03:48 | tsys01 | temperature | 4.64 |
del pf, tdf
# save space
# do pf.info() and tdf.info()
# <class 'pandas.core.frame.DataFrame'>
# DatetimeIndex: 82406931 entries, 2018-03-06 22:14:56 to 2021-03-22 05:18:42
# Data columns (total 3 columns):
# # Column Dtype
# --- ------ -----
# 0 sensor category
# 1 parameter category
# 2 value_hrf float32
# dtypes: category(2), float32(1)
# memory usage: 1.1 GB
Start and end of data:
tsys.head()
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-01-01 00:00:03 | tsys01 | temperature | 4.80 |
| 2019-01-01 00:00:28 | tsys01 | temperature | 4.72 |
| 2019-01-01 00:00:53 | tsys01 | temperature | 4.67 |
| 2019-01-01 00:01:18 | tsys01 | temperature | 4.68 |
| 2019-01-01 00:01:43 | tsys01 | temperature | 4.71 |
tsys.tail()
| sensor | parameter | value_hrf | |
|---|---|---|---|
| timestamp | |||
| 2019-02-28 23:58:15 | tsys01 | temperature | -0.03 |
| 2019-02-28 23:58:40 | tsys01 | temperature | 0.15 |
| 2019-02-28 23:59:05 | tsys01 | temperature | 0.08 |
| 2019-02-28 23:59:30 | tsys01 | temperature | 0.05 |
| 2019-02-28 23:59:55 | tsys01 | temperature | -0.20 |
tsys.dtypes
sensor category parameter category value_hrf float32 dtype: object
Looking at our sensor temperatures
tsys.value_hrf[:10]
timestamp 2019-01-01 00:00:03 4.80 2019-01-01 00:00:28 4.72 2019-01-01 00:00:53 4.67 2019-01-01 00:01:18 4.68 2019-01-01 00:01:43 4.71 2019-01-01 00:02:08 4.68 2019-01-01 00:02:33 4.63 2019-01-01 00:02:58 4.72 2019-01-01 00:03:23 4.68 2019-01-01 00:03:48 4.64 Name: value_hrf, dtype: float32
# CONVERT: C to F
Converting from C to F
# F = C×(9/5)+32
tsys['value_hrf'] = tsys['value_hrf'].apply(lambda x: (9/5)*x + 32)
tsys.value_hrf[:10]
timestamp 2019-01-01 00:00:03 40.640 2019-01-01 00:00:28 40.496 2019-01-01 00:00:53 40.406 2019-01-01 00:01:18 40.424 2019-01-01 00:01:43 40.478 2019-01-01 00:02:08 40.424 2019-01-01 00:02:33 40.334 2019-01-01 00:02:58 40.496 2019-01-01 00:03:23 40.424 2019-01-01 00:03:48 40.352 Name: value_hrf, dtype: float64
%cd D:\GITHUB_Repos\SensorAnalysis\ENTER\results
D:\GITHUB_Repos\SensorAnalysis\ENTER\results
# PLOT: two month window with detailed title going forward (KEEP)
Saving some plots
%cd D:\GITHUB_Repos\SensorAnalysis\ENTER\results
D:\GITHUB_Repos\SensorAnalysis\ENTER\results
Plotting our raw temperature data to see what it looks like
!dir
Volume in drive D is New Volume
Volume Serial Number is A060-E317
Directory of D:\GITHUB_Repos\SensorAnalysis\ENTER\results
09/24/2021 02:20 PM <DIR> .
09/24/2021 02:20 PM <DIR> ..
09/23/2021 01:17 PM 2,690 AoT_unique_nodeids.txt
09/23/2021 08:15 PM 17,387 conda_list_tom.txt
09/24/2021 11:46 AM 1,995,019 dbrun_1.png
09/24/2021 01:56 PM 2,202,989 dbr_1.png
09/24/2021 01:06 PM 1,547,242 dbr_1_r1.png
09/24/2021 02:03 PM 5,526,047 dbr_1_r2.png
09/24/2021 02:09 PM 4,660,539 dbr_1_r2e.png
09/22/2021 09:35 PM 7,178 dbscan_gridsearch_eps_minsamples.txt
09/22/2021 09:41 PM 28,824 dbscan_gridsearch_eps_minsamples_trial2.txt
09/22/2021 09:45 PM 22,896 dbscan_gridsearch_eps_minsamples_trial3.txt
09/22/2021 09:53 PM 184 dbscan_gridsearch_eps_minsamples_trial4.txt
09/22/2021 10:14 PM 45,268 dbscan_gridsearch_eps_minsamples_trial5.txt
09/23/2021 08:33 PM 1,073,524 dbscan_single_subsensor_temp_data_plotted_two_months.png
09/23/2021 08:59 PM 1,312,141 db_run_1_result.png
09/24/2021 12:23 PM 2,017,902 del1.png
09/24/2021 12:24 PM 2,017,902 del2.png
09/24/2021 12:24 PM 2,023,156 del3.png
09/24/2021 12:25 PM 2,009,771 del4.png
09/24/2021 12:15 PM 1,596,455 delta.png
09/24/2021 12:16 PM 1,604,241 delta2.png
09/23/2021 05:04 PM 169,151 holoviews_timeview.pdf
09/23/2021 05:18 PM 49,548 holoviews_window.png
09/23/2021 05:17 PM 100,061 holoview_timerange.svg
09/24/2021 02:01 PM 2,421,009 pow.png
09/24/2021 11:28 AM 1,092,514 single_subsensor_temp_data_plotted_two_months.png
09/23/2021 08:32 PM 39,310 single_subsensor_temp_data_two_month.png
09/23/2021 09:10 PM 1,503,297 single_subsensor_temp_data_two_month_dbscan_clusters.png
09/24/2021 02:20 PM 12,031,828 temp_1q_2019_chicago.html
28 File(s) 47,118,073 bytes
2 Dir(s) 887,074,328,576 bytes free
plt.figure(figsize=(22, 9))
plt.plot(tsys.index, tsys.value_hrf, 'cyan', markersize=1, linewidth=.9, label="temp sensor")
# , 'k.', markersize=8)
# plt.legend(['Temp'])
# plt.xlabel('timestamp')
# plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
plt.ylabel('Temperature F°')
plt.box(False)
# 051 Cottage Grove Ave & 79th St Chicago IL
plt.title(r'Temperature Data - Chicago Sensor (time window: 1Q 2019, lat/lon: 41.751295,87.605288, node location identifier: 001e0610ee36, subsensor identifier: tsys01)')
plt.grid(axis='y', color='grey', linestyle='-', linewidth=.5, alpha=.8)
plt.locator_params(axis='y', nbins=10)
# plt.legend(loc = "upper right")
plt.tight_layout()
plt.show();
# previously saved it here:
# plt.savefig('single_subsensor_temp_data_plotted_two_months.png', dpi=600)
DBSCAN fundamentals
Help on class DBSCAN in module sklearn.cluster._dbscan:
class DBSCAN(sklearn.base.ClusterMixin, sklearn.base.BaseEstimator)
DBSCAN(eps=0.5, *, min_samples=5, metric='euclidean', metric_params=None, algorithm='auto', leaf_size=30, p=None, n_jobs=None)
Perform DBSCAN clustering from vector array or distance matrix.
DBSCAN - Density-Based Spatial Clustering of Applications with Noise.
Finds core samples of high density and expands clusters from them.
Good for data which contains clusters of similar density.
Parameters
----------
eps : float, default=0.5
The maximum distance between two samples for one to be considered
as in the neighborhood of the other. This is not a maximum bound
on the distances of points within a cluster. This is the most
important DBSCAN parameter to choose appropriately for your data set
and distance function.
min_samples : int, default=5
The number of samples (or total weight) in a neighborhood for a point
to be considered as a core point. This includes the point itself.
metric : string, or callable, default='euclidean'
The metric to use when calculating distance between instances in a
feature array. If metric is a string or callable, it must be one of
the options allowed by :func:`sklearn.metrics.pairwise_distances` for
its metric parameter.
If metric is "precomputed", X is assumed to be a distance matrix and
must be square. X may be a :term:`Glossary <sparse graph>`, in which
case only "nonzero" elements may be considered neighbors for DBSCAN.
.. versionadded:: 0.17
metric *precomputed* to accept precomputed sparse matrix.
metric_params : dict, default=None
Additional keyword arguments for the metric function.
.. versionadded:: 0.19
algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, default='auto'
The algorithm to be used by the NearestNeighbors module
to compute pointwise distances and find nearest neighbors.
See NearestNeighbors module documentation for details.
leaf_size : int, default=30
Leaf size passed to BallTree or cKDTree. This can affect the speed
of the construction and query, as well as the memory required
to store the tree. The optimal value depends
on the nature of the problem.
p : float, default=None
The power of the Minkowski metric to be used to calculate distance
between points. If None, then ``p=2`` (equivalent to the Euclidean
distance).
Attributes
----------
core_sample_indices_ : ndarray of shape (n_core_samples,)
Indices of core samples.
components_ : ndarray of shape (n_core_samples, n_features)
Copy of each core sample found by training.
labels_ : ndarray of shape (n_samples)
Cluster labels for each point in the dataset given to fit().
Noisy samples are given the label -1.
Examples
--------
>>> from sklearn.cluster import DBSCAN
>>> import numpy as np
>>> X = np.array([[1, 2], [2, 2], [2, 3],
... [8, 7], [8, 8], [25, 80]])
>>> clustering = DBSCAN(eps=3, min_samples=2).fit(X)
>>> clustering.labels_
array([ 0, 0, 0, 1, 1, -1])
>>> clustering
DBSCAN(eps=3, min_samples=2)
See Also
--------
OPTICS : A similar clustering at multiple values of eps. Our implementation
is optimized for memory usage.
Method resolution order:
DBSCAN
sklearn.base.ClusterMixin
sklearn.base.BaseEstimator
builtins.object
fit(self, X, y=None, sample_weight=None)
Perform DBSCAN clustering from features, or distance matrix.
Parameters
----------
X : {array-like, sparse matrix} of shape (n_samples, n_features), or
(n_samples, n_samples)
Training instances to cluster, or distances between instances if
``metric='precomputed'``. If a sparse matrix is provided, it will
be converted into a sparse ``csr_matrix``.
sample_weight : array-like of shape (n_samples,), default=None
Weight of each sample, such that a sample with a weight of at least
``min_samples`` is by itself a core sample; a sample with a
negative weight may inhibit its eps-neighbor from being core.
Note that weights are absolute, and default to 1.
y : Ignored
Not used, present here for API consistency by convention.
Converting raw data via StandardScaler()
tsys.columns
Index(['sensor', 'parameter', 'value_hrf'], dtype='object')
# PYTHONIC: kill a column IF it exits already
# (so no warning/errrors stopping us)
thisFilter = tsys.filter('cluster')
tsys.drop(thisFilter, inplace=True, axis=1)
# tsys.drop('cluster', axis=1, inplace=True)
# if you iterate, you need to remove !
tsys.columns.values
array(['sensor', 'parameter', 'value_hrf'], dtype=object)
X = tsys['value_hrf'].copy()
# X[:10]
# X.values[:10]
# Utilize standard scaler
X = StandardScaler().fit_transform(X.values.reshape(-1,1))
# X[:10] # normalized ! ! ! !
# array([[0.82499978],
# [0.81444817],
# [0.80785348],
# [0.80917239],
# [0.81312926],
# [0.80917239],
# [0.80257771],
# [0.81444817],
# [0.80917239],
# [0.80389662]])
plt.style.use('default')
plt.figure(figsize=(25, 7))
plt.plot(tsys.index, X, 'grey', markersize=1, linewidth=.9)
# plt.legend()
plt.box(False)
print("Standard Scaler View of Data (normalized):")
plt.show();
# normalized
Standard Scaler View of Data (normalized):
-- ITERATION 1 --
Running DBSCAN on the raw data to determine clusters of time series parameter values
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
# Previously tuned parameters, but for now really just looking around
myeps = .01
myminsamples = 25
#************************************************************************************
db = DBSCAN(eps=myeps, min_samples=myminsamples, metric='euclidean', n_jobs=-1).fit(X)
#************************************************************************************
# db.get_params(deep=True)
# {'algorithm': 'auto',
# 'eps': 0.01,
# 'leaf_size': 30,
# 'metric': 'euclidean',
# 'metric_params': None,
# 'min_samples': 25,
# 'n_jobs': -1,
# 'p': None}
label=db.labels_
#identifying the points which makes up our core points
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
# Calculating the number of clusters
n_clusters=len(set(label))- (1 if -1 in label else 0)
print(myeps,myminsamples)
print('\nNo of clusters:',n_clusters, '\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
# tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
0.01 25 No of clusters: 4 0 198355 3 4943 -1 141 1 30 2 12 Name: cluster, dtype: int64
-- ITERATION 1 --
Plotting the clusters that DBSCAN identified, with our base eps and min_samples hyperparameters:
# AAA. DO NOT LOSE THIS CODE, WORKING !
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.title('Sensor Temperature Data Anomalies, 1Q-2019, node location identifier: 001e0610ee36, subsensor identifier: tsys01, lat/lon: 41.751295,87.605288, [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples),
fontsize=14)
# plt.legend()
plt.legend(loc='upper center', fontsize=15)
plt.tight_layout()
plt.box(False)
plt.show();
# these results have been saved, comment out now be careful
# plt.savefig('dbr_1.png', dpi=800);
-- ITERATION 1 --
Translating / Understanding what we are seeing with anomaly detection:
# BBB. DO NOT LOSE THIS CODE, WORKING
# plt.style.use('seaborn-ticks')
plt.style.use('default')
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
fig, ax = plt.subplots(figsize=(30,8))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# plt.style.use('white_background')
plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temperature readings")
# --- original code
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# c = tsys.loc[tsys['cluster'] == 1].index
# d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# e = tsys.loc[tsys['cluster'] == 2].index
# f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# g = tsys.loc[tsys['cluster'] == 3].index
# h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="gray"),
# xytext=(10, -40), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
ax.annotate("IDENTIFIED: UNUSUALLY HIGH WINTER TEMPS", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="red"),
xytext=(150,0), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
ax.annotate("POLAR VORTEX STORM HITS CHICAGO !", xy=(pd.Timestamp('2019-01-30'), -13), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="blue"),
xytext=(-140, 0), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
ax.annotate("'Predecessor' Triggers", xy=(pd.Timestamp('2019-02-01'), -7), xycoords='data',
bbox=dict(boxstyle="round", fc="none", ec="yellow"),
xytext=(125, 25), textcoords='offset points', ha='center',
arrowprops=dict(arrowstyle="->"))
# https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
plt.title('Chicago Sensor Data Anomalies (Unsupervised Machine Learning): timerange: 1Q-2019, node location identifier: 001e0610ee36, subsensor identifier: tsys01, address: Cottage Grove Ave & 79th St Chicago IL, lat/lon: 41.751295,87.605288, [alg=DBSCAN, eps={0}, min_samples={1}, cluster_count:4]'.format(myeps, myminsamples),
fontsize=12)
plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
plt.locator_params(axis='y', nbins=10)
plt.locator_params(axis='x', nbins=20)
plt.legend(loc='upper center', fontsize=12)
plt.ylabel('Temperature F°')
plt.box(False)
plt.tight_layout()
plt.show();
# plt.savefig('pow.png', dpi=800) # single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# plt.savefig('delta2.png', facecolor = 'white', dpi=600)
# plt.savefig('temp2.png', facecolor="black", edgecolor="none")
-- ITERATION 1 --
Visualizing some closeup ranges (red anomaly and blue anomaly regions):
# ccc. WINDOWS AND RANGES
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 6))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1.5, linewidth=1.6)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=52, alpha=0.66)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# ax.annotate("POLAR VORTEX STORM HITS CHICAGO !", xy=(pd.Timestamp('2019-01-05 21:00:00'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="blue"),
# xytext=(-140, 0), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.title('CLOSEUP (January 5th, 8:04 PM - 9:40 PM) - Anomalous High Temperature Readings Identified via Unsupervised Machine Learning (DBSCAN) [Chicago Sensor 1, 25s granularity]'.format(myeps, myminsamples),
fontsize=14)
# plt.legend()
# plt.legend(loc='center', fontsize=15)
plt.xlim(pd.Timestamp('2019-01-05 20:04:00'), pd.Timestamp('2019-01-05 21:41:00'))
plt.ylim((50, 66))
# @#pd.Timestamp('2019-01-05 19:59:20'), pd.Timestamp('2019-01-05 21:26:33'))
plt.tight_layout()
plt.box(False)
plt.show();
# these results have been saved, comment out now be careful
# plt.savefig('dbr_1_r1.png', dpi=800);
# secondary window
plt.style.use('seaborn-ticks')
plt.figure(figsize=(24, 11))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1.5, linewidth=.7)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=52, alpha=0.66)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
plt.scatter(c, d, color='blue', label = 'Cluster 3', s=5, alpha=0.6)
e = tsys.loc[tsys['cluster'] == 1].index
f= tsys[(tsys['cluster'] == 1)]['value_hrf']
plt.scatter(e, f, color='green', label = 'Cluster 1', s=50, alpha=0.6)
g = tsys.loc[tsys['cluster'] == 2].index
h= tsys[(tsys['cluster'] == 2)]['value_hrf']
plt.scatter(g, h, color='yellow', label = 'Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
plt.title('CLOSEUP (January 30th-31st, 2019) - Anomalous Low Temperature Readings Identified via Unsupervised Machine Learning (DBSCAN) [Chicago Sensor 1, 25s granularity]'.format(myeps, myminsamples),
fontsize=14)
# plt.legend()
plt.legend(loc='lower left', fontsize=15)
plt.xlim(pd.Timestamp('2019-01-30 05:09:00'), pd.Timestamp('2019-01-31 17:01:00'))
plt.ylim((-26,-2))
# @#pd.Timestamp('2019-01-05 19:59:20'), pd.Timestamp('2019-01-05 21:26:33'))
plt.tight_layout()
plt.box(False)
plt.show();
# these results have been saved, comment out now be careful
# plt.savefig('dbr_1_r2e.png', dpi=700);
As a reference, we use plotly to show all the temperature values over the entire range,
so readers can home in on the window they wish to view/explore...
import plotly.express as px
# fig = px.scatter(tsys, x=tsys.index,
# y=tsys.value_hrf,
# title="1Q 2019 - Chicago Sensor 1 - Raw Data (Temperature in F)",
# template = "plotly_dark").update_traces(marker=dict(size=1,
# line=dict(width=.5,
# color='darkgrey')),
# selector=dict(mode='markers')).update_layout(yaxis_title="Temperature F°")
# fig.write_html("temp_1q_2019_chicago.html")
# fig.show()
# Uncomment everything out, and it will show the following (via url):
-- ITERATION 1 --
Outputting and exporting the actual anomalous values (ranges), and plotting shaded regions...
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
print('\nANOMALOUS REGION ("RED", Cluster -1):')
print('(can export to .txt if desired)')
# for date in a.values:
# print(a)
for c in range(140):
print(a[c])
# print(a[0])
# print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
print('Red Window Start:', min_a)
print('Red Window End: ', max_a)
print("")
plt.axvspan(min_a, max_a, alpha=.3, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
print('Dominant Blue Window Start:', min_c)
print('Dominant Blue Window End: ', max_c)
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.3, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend(loc='upper center', borderaxespad=0.)
plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-05 20:07:20'), pd.Timestamp('2019-01-31 16:26:33'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# plt.savefig('dbrun_1.png', dpi=800);
ANOMALOUS REGION ("RED", Cluster -1):
(can export to .txt if desired)
2019-01-05 20:07:20
2019-01-05 20:07:45
2019-01-05 20:08:11
2019-01-05 20:08:36
2019-01-05 20:09:01
2019-01-05 20:09:26
2019-01-05 20:09:51
2019-01-05 20:10:16
2019-01-05 20:10:41
2019-01-05 20:21:57
2019-01-05 20:22:22
2019-01-05 20:27:23
2019-01-05 20:27:48
2019-01-05 20:28:13
2019-01-05 20:28:38
2019-01-05 20:29:03
2019-01-05 20:29:28
2019-01-05 20:31:58
2019-01-05 20:32:23
2019-01-05 20:33:39
2019-01-05 20:34:04
2019-01-05 20:34:29
2019-01-05 20:34:54
2019-01-05 20:35:19
2019-01-05 20:35:44
2019-01-05 20:36:09
2019-01-05 20:36:34
2019-01-05 20:36:59
2019-01-05 20:37:24
2019-01-05 20:37:49
2019-01-05 20:38:14
2019-01-05 20:38:39
2019-01-05 20:39:04
2019-01-05 20:39:29
2019-01-05 20:39:54
2019-01-05 20:40:19
2019-01-05 20:40:44
2019-01-05 20:41:09
2019-01-05 20:41:34
2019-01-05 20:41:59
2019-01-05 20:42:25
2019-01-05 20:42:50
2019-01-05 20:43:15
2019-01-05 20:43:40
2019-01-05 20:44:05
2019-01-05 20:44:30
2019-01-05 20:44:55
2019-01-05 20:45:20
2019-01-05 20:45:45
2019-01-05 20:46:10
2019-01-05 20:46:35
2019-01-05 20:47:00
2019-01-05 20:47:25
2019-01-05 20:47:50
2019-01-05 20:48:15
2019-01-05 20:48:40
2019-01-05 20:49:05
2019-01-05 20:49:31
2019-01-05 20:49:56
2019-01-05 20:50:21
2019-01-05 20:50:46
2019-01-05 20:51:11
2019-01-05 20:51:36
2019-01-05 20:52:01
2019-01-05 20:52:26
2019-01-05 20:52:51
2019-01-05 20:53:16
2019-01-05 20:53:41
2019-01-05 20:54:06
2019-01-05 20:54:31
2019-01-05 20:54:56
2019-01-05 20:55:21
2019-01-05 20:55:46
2019-01-05 20:56:11
2019-01-05 20:56:36
2019-01-05 20:57:01
2019-01-05 20:57:26
2019-01-05 20:57:51
2019-01-05 20:58:16
2019-01-05 20:58:42
2019-01-05 20:59:07
2019-01-05 20:59:32
2019-01-05 20:59:57
2019-01-05 21:00:22
2019-01-05 21:00:47
2019-01-05 21:01:12
2019-01-05 21:01:37
2019-01-05 21:02:02
2019-01-05 21:02:27
2019-01-05 21:02:52
2019-01-05 21:03:17
2019-01-05 21:03:42
2019-01-05 21:04:07
2019-01-05 21:04:32
2019-01-05 21:04:57
2019-01-05 21:05:22
2019-01-05 21:05:47
2019-01-05 21:06:12
2019-01-05 21:06:38
2019-01-05 21:07:03
2019-01-05 21:07:28
2019-01-05 21:07:53
2019-01-05 21:08:18
2019-01-05 21:08:43
2019-01-05 21:09:08
2019-01-05 21:09:33
2019-01-05 21:09:58
2019-01-05 21:10:23
2019-01-05 21:10:48
2019-01-05 21:11:13
2019-01-05 21:11:38
2019-01-05 21:12:03
2019-01-05 21:12:28
2019-01-05 21:12:53
2019-01-05 21:13:18
2019-01-05 21:13:43
2019-01-05 21:14:08
2019-01-05 21:14:33
2019-01-05 21:14:58
2019-01-05 21:15:24
2019-01-05 21:15:49
2019-01-05 21:16:14
2019-01-05 21:16:39
2019-01-05 21:17:04
2019-01-05 21:24:35
2019-01-05 21:25:00
2019-01-05 21:25:25
2019-01-05 21:25:50
2019-01-05 21:35:26
2019-01-05 21:38:21
2019-01-05 21:38:46
2019-01-05 21:39:11
2019-01-05 21:39:36
2019-01-30 05:38:30
2019-01-30 05:38:55
2019-01-30 05:41:50
2019-01-30 05:44:20
2019-01-30 05:46:01
2019-01-30 05:47:41
2019-01-30 05:48:06
Red Window Start: 2019-01-05 20:07:20
Red Window End: 2019-01-05 21:39:36
ARCTIC POLAR VORTEX:
Dominant Blue Window Start: 2019-01-30 05:49:46
Dominant Blue Window End: 2019-01-31 16:26:33
# # BBB. DO NOT LOSE THIS CODE, WORKING
# plt.style.use('seaborn-ticks')
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# # plt.style.use('white_background')
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
# # --- original code
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# # c = tsys.loc[tsys['cluster'] == 1].index
# # d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# # g = tsys.loc[tsys['cluster'] == 3].index
# # h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(130,20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(-140, 20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc='upper center', borderaxespad=0.)
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# plt.show();
# # plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
print('\nANOMALOUS REGION (Cluster: -1):')
# for date in a.values:
# print(a)
# for c in range(140):
# halt for now print(a[c])
# print(a[0])
# print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
print('Red Window Start:', min_a)
print('Red Window End: ', max_a)
print('Plotting below:')
print("")
plt.axvspan(min_a, max_a, alpha=.2, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
print('\nANOMALOUS REGION (Cluster: 1):\n')
print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
print('Dominant Blue Window Start:', min_c)
print('Dominant Blue Window End: ', max_c)
print('Plotting second below:\n')
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.3, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# plt.legend(loc='upper center', borderaxespad=1)
# plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-05 19:55:00'), pd.Timestamp('2019-01-05 22:00:00'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# plt.savefig('dbrun_1.png', dpi=800);
ANOMALOUS REGION (Cluster: -1): Red Window Start: 2019-01-05 20:07:20 Red Window End: 2019-01-05 21:39:36 Plotting below: ANOMALOUS REGION (Cluster: 1): ARCTIC POLAR VORTEX: Dominant Blue Window Start: 2019-01-30 05:49:46 Dominant Blue Window End: 2019-01-31 16:26:33 Plotting second below:
# adding shade !!!!!!!!!!!!
plt.style.use('seaborn-ticks')
plt.figure(figsize=(25, 10))
plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
a = tsys.loc[tsys['cluster'] == -1].index
b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# print('\nANOMALOUS REGION ("RED", Cluster -1):')
# for date in a.values:
# print(a)
# for c in range(140):
# print(a[c])
# # print(a[0])
# # print(a[1])
min_a = a.min()
max_a = a[-9]
print("")
# print('Red Window Start:', min_a)
# print('Red Window End: ', max_a)
print("")
plt.axvspan(min_a, max_a, alpha=.3, color='red')
# line1 = min_a
# line2 = max_a
# ax.axvspan(line1, line2, alpha=.7, color='red')
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
c = tsys.loc[tsys['cluster'] == 3].index
d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# print('ARCTIC POLAR VORTEX:')
min_c = c.min()
max_c = c.max()
# print('Dominant Blue Window Start:', min_c)
# print('Dominant Blue Window End: ', max_c)
line1 = min_c
line2 = max_c
plt.axvspan(line1, line2, alpha=.2, color='blue')
# plt.axvline(x=0.22058956)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
plt.ylabel('Temperature F°')
plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
plt.tight_layout()
plt.box(False)
plt.xlim(pd.Timestamp('2019-01-29 23:07:20'), pd.Timestamp('2019-02-01 00:26:33'))
plt.ylim( (-30, 10))
# pd.Timestamp('2019-01-29 23:07:20'), pd.Timestamp('2019-02-01 00:26:33'))
# ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
plt.show();
# miniplot if you wanted:
# plt.figure(figsize=(20, 20))
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# plt.show();
# # clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# # reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
# myeps = .01
# myminsamples = 25
# #************************************************************************************
# db = DBSCAN(eps=myeps, min_samples=myminsamples, metric='euclidean', n_jobs=-1).fit(X)
# #************************************************************************************
# # db.get_params(deep=True)
# # {'algorithm': 'auto',
# # 'eps': 0.01,
# # 'leaf_size': 30,
# # 'metric': 'euclidean',
# # 'metric_params': None,
# # 'min_samples': 25,
# # 'n_jobs': -1,
# # 'p': None}
# label=db.labels_
# #identifying the points which makes up our core points
# sample_cores=np.zeros_like(label,dtype=bool)
# sample_cores[db.core_sample_indices_]=True
# # Calculating the number of clusters
# n_clusters=len(set(label))- (1 if -1 in label else 0)
# print(myeps,myminsamples)
# print('\nNo of clusters:',n_clusters, '\n')
# # print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
# tsys["cluster"] = db.labels_
# print(tsys["cluster"].value_counts(), '\n')
# #tsys["cluster"] = db.labels_
# # tsys["cluster"].value_counts()
# # tsys["cluster"] = db.labels_
# # tsys["cluster"].value_counts()
# plt.style.use('seaborn-ticks')
# plt.figure(figsize=(25, 8))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # plt.xlabel('Arrival Date')
# plt.ylabel('Temperature F°')
# # plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.title('Temp Data Anomalies, eps = {}'.format(myeps))
# plt.legend()
# plt.tight_layout()
# plt.box(False)
# plt.show();
# # use dark background !!!!!!!!!!!!!!!!!!!!!!
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.style.use('white_background')
# # plt.figure(figsize=(30, 8))
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# c = tsys.loc[tsys['cluster'] == 1].index
# d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# e = tsys.loc[tsys['cluster'] == 2].index
# f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# g = tsys.loc[tsys['cluster'] == 3].index
# h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(130,20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(-140, 20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc = "upper left")
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# # plt.show();
# plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# plt.figure(figsize=(20, 6))
# plt.scatter(g, h, color='grey', label = 'Anomaly Cluster 3', s=5, alpha=0.5)
# plt.show();
# # db = DBSCAN(eps=0.01, min_samples=25, metric='euclidean', n_jobs=-1).fit(X)
# plt.figure(figsize=(35, 50))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# c = tsys.loc[tsys['cluster'] == 1].index
# d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# e = tsys.loc[tsys['cluster'] == 2].index
# f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# g = tsys.loc[tsys['cluster'] == 3].index
# h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(e, f, color='orange', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# plt.scatter(g, h, color='black', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# # plt.xlabel('Arrival Date')
# plt.ylabel('Temperature C°')
# plt.legend()
# plt.box(False)
# plt.show();
# The lowest temperature at Chicago was -23 degrees observed at Chicago O'Hare International Airport on January 30th.
# The lowest temperature at Rockford was -31 degrees observed at Chicago Rockford International Airport on January 31st.
# The lowest temperature at NWS Chicago in Romeoville was -24 degrees. This ties the record set on January 16, 2009.
# The lowest temperature at Chicago Midway Airport 3SW observed was -22 degrees on January 31, 2019.
# plt.figure(figsize=(25, 5))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# c = tsys.loc[tsys['cluster'] == 1].index
# d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=35, alpha=0.7)
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=35, alpha=0.7)
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=30, alpha=0.5)
# # plt.xlabel('Arrival Date')
# plt.ylabel('Temperature C°')
# plt.legend()
# plt.box(False)
# plt.show();
# ORIGINAL CODE BEFORE I USED THE NEW ABOVE
# label=db.labels_
# from sklearn import metrics
# #identifying the points which makes up our core points
# sample_cores=np.zeros_like(label,dtype=bool)
# sample_cores[db.core_sample_indices_]=True
# # Calculating the number of clusters
# n_clusters=len(set(label))- (1 if -1 in label else 0)
# print('No of clusters:',n_clusters)
# core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
# core_samples_mask[db.core_sample_indices_] = True
# labels = db.labels_
# # Number of clusters in labels, ignoring noise if present.
# n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
# n_noise_ = list(labels).count(-1)
# print('Estimated number of clusters: %d' % n_clusters_)
# # print('Estimated number of noise points: %d' % n_noise_)
# # print("Homogeneity: %0.3f" % metrics.homogeneity_score(labels_true, labels))
# # print("Completeness: %0.3f" % metrics.completeness_score(labels_true, labels))
# # print("V-measure: %0.3f" % metrics.v_measure_score(labels_true, labels))
# print("Adjusted Rand Index: %0.3f"
# % metrics.adjusted_rand_score(labels_true, labels))
# print("Adjusted Mutual Information: %0.3f"
# % metrics.adjusted_mutual_info_score(labels_true, labels))
# print("Silhouette Coefficient: %0.3f"
# % metrics.silhouette_score(X, labels))
# from sklearn.neighbors import NearestNeighbors
# nearest_neighbors = NearestNeighbors(n_neighbors=11)
# neighbors = nearest_neighbors.fit(X)
# distances, indices = neighbors.kneighbors(X)
# distances = np.sort(distances[:,10], axis=0)
# fig = plt.figure(figsize=(5, 5))
# plt.plot(distances)
# plt.xlabel("Points")
# plt.ylabel("Distance")
# plt.savefig("Distance_curve.png", dpi=300)
from sklearn.cluster import DBSCAN
clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(ts_dataframe['Normalized Profit']).reshape(-1,1))
labels = clustering1.labels_
outlier_pos = np.where(labels == -1)[0]
x = []; y = [];
for pos in outlier_pos:
x.append(np.array(ts_dataframe['Normalized Profit'])[pos])
y.append(ts_dataframe['Normalized Profit'].index[pos])
plt.plot(ts_dataframe['Normalized Profit'].loc[ts_dataframe['Normalized Profit'].index], 'k-')
plt.plot(y,x,'r*', markersize=8)
plt.legend(['Actual', 'Anomaly Detected'])
plt.xlabel('Time Period')
plt.xticks([0, 20, 40, 60, 80, 99],[ts_dataframe.index[0],ts_dataframe.index[20], ts_dataframe.index[40], ts_dataframe.index[60], ts_dataframe.index[80], ts_dataframe.index[99]] ,rotation=45)
plt.ylabel('Normalized Profit')
# # Sets the figure size temporarily but has to be set again the next plot
# plt.figure(figsize=(20,16))
# sns.lineplot(x=tdf.index, y="value_hrf", hue="sensor", data=tdf);
# plt.show()
# Sets the figure size temporarily but has to be set again the next plot
# plt.figure(figsize=(20,16))
# sns.lineplot(x=tdf.index, y="value_hrf", hue="sensor", data=tdf);
# plt.show()
# this time, i WONT use the standard scaler, just to see what we see...
# clustering1 = DBSCAN(eps=0.09, min_samples=6).fit(np.array(tsys['value_hrf']).
# reshape(-1,1)) # or you could take pandas series and go to df with to_frame() pandas function
from sklearn.cluster import DBSCAN
# eps=0.2, min_samples=8
# The argument 'eps' is the distance between two samples to be considered as a neighborhood and
# 'min_samples' is the number of samples in a neighborhood
####################################################################
db = DBSCAN(eps=0.1, min_samples=10, metric='euclidean', n_jobs=-1)
####################################################################
# X = StandardScaler().fit_transform(X)
# db = DBSCAN(eps=0.3, min_samples=10).fit(X)
db = db.fit(np.array(tsys['value_hrf']).reshape(-1, 1)) # i think smarter to do to_frame() instead...
label=db.labels_
from sklearn import metrics
#identifying the points which makes up our core points
sample_cores=np.zeros_like(label,dtype=bool)
sample_cores[db.core_sample_indices_]=True
# Calculating the number of clusters
n_clusters=len(set(label))- (1 if -1 in label else 0)
print('\nNo of clusters:',n_clusters, '\n\n')
# print('\nRemember, in theory 1 cluster actually means one normal cluster and everything else is _noise_')
tsys["cluster"] = db.labels_
print(tsys["cluster"].value_counts(), '\n')
#tsys["cluster"] = db.labels_
# tsys["cluster"].value_counts()
RESULTS:
========
db = DBSCAN(eps=0.03, min_samples=6, metric='euclidean', n_jobs=-1)
0 139811
3 4950
2 37
1 26
-1 19
4 10
db = DBSCAN(eps=0.04, min_samples=5, metric='euclidean', n_jobs=-1)
0 144840
-1 13
db = DBSCAN(eps=0.04, min_samples=7, metric='euclidean', n_jobs=-1)
0 139848
1 4989
-1 16
db = DBSCAN(eps=0.04, min_samples=7, metric='euclidean', n_jobs=-1)
0 139848
1 4989
-1 16
db = DBSCAN(eps=0.1, min_samples=10, metric='euclidean', n_jobs=-1)
0 198364
4 4990
1 66
-1 41
2 10
3 10
``
# ddd. WINDOWS AND RANGES
# plt.style.use('seaborn-ticks')
# plt.figure(figsize=(25, 6))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1.5, linewidth=1.6)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=52, alpha=0.66)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# # ax.annotate("POLAR VORTEX STORM HITS CHICAGO !", xy=(pd.Timestamp('2019-01-05 21:00:00'), 60), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="blue"),
# # xytext=(-140, 0), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.ylabel('Temperature F°')
# # plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.title('CLOSEUP (January 5th, 8:04 PM - 9:40 PM) - Anomalous High Temperature Readings Identified via Unsupervised Machine Learning (DBSCAN) [Chicago Sensor 1, 25s granularity]'.format(myeps, myminsamples),
# fontsize=14)
# # plt.legend()
# # plt.legend(loc='center', fontsize=15)
# plt.xlim(pd.Timestamp('2019-01-05 20:05:00'), pd.Timestamp('2019-01-05 21:41:00'))
# plt.ylim((50, 66))
# # @#pd.Timestamp('2019-01-05 19:59:20'), pd.Timestamp('2019-01-05 21:26:33'))
# plt.tight_layout()
# plt.box(False)
# # plt.show();
# # these results have been saved, comment out now be careful
# plt.savefig('dbr_1_r1.png', dpi=800);
# # CCC. CLOSEUP VISUALS, DO NOT LOSE THIS CODE, WORKING
# # plt.style.use('seaborn-ticks')
# plt.style.use('default')
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# # plt.style.use('white_background')
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temperature readings")
# # --- original code
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# # c = tsys.loc[tsys['cluster'] == 1].index
# # d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# # g = tsys.loc[tsys['cluster'] == 3].index
# # h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
# # c = tsys.loc[tsys['cluster'] == 3].index
# # d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# # e = tsys.loc[tsys['cluster'] == 1].index
# # f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# # g = tsys.loc[tsys['cluster'] == 2].index
# # h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("IDENTIFIED: UNUSUALLY HIGH WINTER TEMPS", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(150,0), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("POLAR VORTEX STORM HITS CHICAGO !", xy=(pd.Timestamp('2019-01-30'), -13), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="blue"),
# xytext=(-140, 0), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("'Predecessor' Triggers", xy=(pd.Timestamp('2019-02-01'), -7), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(125, 25), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title('Chicago Sensor Data Anomalies (Unsupervised Machine Learning): timerange: 1Q-2019, node location identifier: 001e0610ee36, subsensor identifier: tsys01, address: Cottage Grove Ave & 79th St Chicago IL, lat/lon: 41.751295,87.605288, [alg=DBSCAN, eps={0}, min_samples={1}, cluster_count:4]'.format(myeps, myminsamples),
# fontsize=12)
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc='upper center', fontsize=12)
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# plt.show();
# # plt.savefig('pow.png', dpi=700) # single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# # plt.savefig('delta2.png', facecolor = 'white', dpi=600)
# # plt.savefig('temp2.png', facecolor="black", edgecolor="none")
# # BBB. DO NOT LOSE THIS CODE, WORKING
# # plt.style.use('seaborn-ticks')
# plt.style.use('default')
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# # plt.style.use('white_background')
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temperature readings")
# # --- original code
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# # c = tsys.loc[tsys['cluster'] == 1].index
# # d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# # g = tsys.loc[tsys['cluster'] == 3].index
# # h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
# # c = tsys.loc[tsys['cluster'] == 3].index
# # d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# # e = tsys.loc[tsys['cluster'] == 1].index
# # f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# # g = tsys.loc[tsys['cluster'] == 2].index
# # h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("IDENTIFIED: UNUSUALLY HIGH WINTER TEMPS", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(150,0), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("POLAR VORTEX STORM HITS CHICAGO !", xy=(pd.Timestamp('2019-01-30'), -13), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="blue"),
# xytext=(-140, 0), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("'Predecessor' Triggers", xy=(pd.Timestamp('2019-02-01'), -7), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(125, 25), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title('Chicago Sensor Data Anomalies (Unsupervised Machine Learning): timerange: 1Q-2019, node location identifier: 001e0610ee36, subsensor identifier: tsys01, address: Cottage Grove Ave & 79th St Chicago IL, lat/lon: 41.751295,87.605288, [alg=DBSCAN, eps={0}, min_samples={1}, cluster_count:4]'.format(myeps, myminsamples),
# fontsize=12)
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc='upper center', fontsize=12)
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# plt.show();
# # plt.savefig('pow.png', dpi=700) # single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# # plt.savefig('delta2.png', facecolor = 'white', dpi=600)
# # plt.savefig('temp2.png', facecolor="black", edgecolor="none")
# # BBB. DO NOT LOSE THIS CODE, WORKING
# plt.style.use('seaborn-ticks')
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# # plt.style.use('white_background')
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
# # --- original code
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# # c = tsys.loc[tsys['cluster'] == 1].index
# # d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# # g = tsys.loc[tsys['cluster'] == 3].index
# # h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(130,20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(-140, 20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc='upper center', borderaxespad=0.)
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# plt.show();
# # plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# # use dark background !!!!!!!!!!!!!!!!!!!!!!
# plt.style.use('seaborn-ticks')
# fig, ax = plt.subplots(figsize=(30,8))
# # plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# # plt.style.use('white_background')
# plt.plot(tsys.index, tsys.value_hrf, color = 'grey', alpha = .7, markersize=.8, linewidth=.9, label="temp sensor")
# # --- original code
# # a = tsys.loc[tsys['cluster'] == -1].index
# # b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=20, alpha=0.9)
# # c = tsys.loc[tsys['cluster'] == 1].index
# # d= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 1', s=20, alpha=0.9)
# # e = tsys.loc[tsys['cluster'] == 2].index
# # f= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(e, f, color='orange', label = 'Anomaly Cluter 2', s=20, alpha=0.9)
# # g = tsys.loc[tsys['cluster'] == 3].index
# # h= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 3', s=20, alpha=0.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# plt.scatter(a, b, color='red', label = 'Anomaly Cluster -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# e = tsys.loc[tsys['cluster'] == 1].index
# f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# plt.scatter(e, f, color='green', label = 'Anomaly Cluster 1', s=50, alpha=0.6)
# g = tsys.loc[tsys['cluster'] == 2].index
# h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# # ax.annotate('local maximum', xy=('2019-01-01', 10), xytext=('2019-01-29', 5),
# # color="white", arrowprops=dict(facecolor='green', shrink=0.05))
# # ax.annotate("Independence Day", xy=('2019-01-29', -20), xycoords='data',
# # bbox=dict(boxstyle="round", fc="none", ec="gray"),
# # xytext=(10, -40), textcoords='offset points', ha='center',
# # arrowprops=dict(arrowstyle="->"))
# ax.annotate("Unusually High Winter Temps", xy=(pd.Timestamp('2019-01-06'), 60), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="red"),
# xytext=(130,20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# ax.annotate("Polar Vortex Storm Hits Chicago !", xy=(pd.Timestamp('2019-01-30'), -12), xycoords='data',
# bbox=dict(boxstyle="round", fc="none", ec="yellow"),
# xytext=(-140, 20), textcoords='offset points', ha='center',
# arrowprops=dict(arrowstyle="->"))
# # https://jakevdp.github.io/PythonDataScienceHandbook/04.09-text-and-annotation.html
# plt.title(r'Temperature Data Anomalies - Chicago Sensor (alg: DBSCAN, time window: 2 months, sensor node identifier: 001e0610ee36, subsensor: tsys01)')
# plt.grid(axis='y', color='grey', linestyle='--', linewidth=.5, alpha=.3)
# plt.locator_params(axis='y', nbins=10)
# plt.locator_params(axis='x', nbins=20)
# plt.legend(loc='upper center', borderaxespad=0.)
# plt.ylabel('Temperature F°')
# plt.box(False)
# plt.tight_layout()
# plt.show();
# # plt.savefig('single_subsensor_temp_data_two_month_dbscan_clusters.png', dpi=600)
# # adding shade !!!!!!!!!!!!
# plt.style.use('seaborn-ticks')
# plt.figure(figsize=(25, 10))
# plt.plot(tsys.index, tsys.value_hrf, 'grey', markersize=1, linewidth=.9)
# a = tsys.loc[tsys['cluster'] == -1].index
# b= tsys[(tsys['cluster'] == -1)]['value_hrf']
# print('\nANOMALOUS REGION (-1):')
# # for date in a.values:
# # print(a)
# for c in range(140):
# print(a[c])
# # print(a[0])
# # print(a[1])
# min_a = a.min()
# max_a = a[-9]
# print("")
# print('Red Window Start:', min_a)
# print('Red Window End: ', max_a)
# print("")
# plt.axvspan(min_a, max_a, alpha=.3, color='red')
# # line1 = min_a
# # line2 = max_a
# # ax.axvspan(line1, line2, alpha=.7, color='red')
# # plt.scatter(a, b, color='red', label = 'Anomaly Cluter -1', s=50, alpha=0.6)
# c = tsys.loc[tsys['cluster'] == 3].index
# d= tsys[(tsys['cluster'] == 3)]['value_hrf']
# # plt.scatter(c, d, color='blue', label = 'Anomaly Cluster 3', s=50, alpha=0.6)
# print('ARCTIC POLAR VORTEX:')
# min_c = c.min()
# max_c = c.max()
# print('Dominant Blue Window Start:', min_c)
# print('Dominant Blue Window End: ', max_c)
# line1 = min_c
# line2 = max_c
# plt.axvspan(line1, line2, alpha=.3, color='blue')
# # plt.axvline(x=0.22058956)
# # e = tsys.loc[tsys['cluster'] == 1].index
# # f= tsys[(tsys['cluster'] == 1)]['value_hrf']
# # plt.scatter(e, f, color='green', label = 'Anomaly Cluter 1', s=50, alpha=0.6)
# # g = tsys.loc[tsys['cluster'] == 2].index
# # h= tsys[(tsys['cluster'] == 2)]['value_hrf']
# # plt.scatter(g, h, color='yellow', label = 'Anomaly Cluster 2', s=50, alpha=0.6)
# plt.ylabel('Temperature F°')
# plt.title('Temp Data Anomalies 1Q-2019 [alg=DBSCAN, eps={0}, min_samples={1}]'.format(myeps, myminsamples))
# # plt.legend(loc='upper center', borderaxespad=0.)
# plt.tight_layout()
# plt.box(False)
# plt.xlim(pd.Timestamp('2019-01-05 20:07:20'), pd.Timestamp('2019-01-31 16:26:33'))
# # ax.set_xlim(pd.Timestamp('2015-02-15'), pd.Timestamp('2015-07-01'))
# plt.show();
# # plt.savefig('dbrun_1.png', dpi=800);